{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "744037f8-6aa9-48f4-b240-a16d13655b98",
   "metadata": {},
   "source": [
    "(exithandler-workflow)=\n",
    "# Running a multiple function workflow with ExitHandler\n",
    "\n",
    "This example uses the Kubernetes function ExitHandler to run multiple functions before continuing to the next function. \n",
    "The ExitHandler function is especially useful to do something that needs to be performed as the very last step; for example, sending an email or updating the run results of the previous steps somewhere else."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "82a7e23c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2025-05-06 08:14:15,634 [info] Server and client versions are not the same but compatible: {'parsed_server_version': Version(major=1, minor=8, patch=0, prerelease='rc53', build=None), 'parsed_client_version': Version(major=1, minor=6, patch=4, prerelease=None, build=None)}\n"
     ]
    }
   ],
   "source": [
    "import mlrun"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e70de79",
   "metadata": {},
   "source": [
    "## Create the project and its functions\n",
    "\n",
    "**Function B is designed to fail, to show the ExitHandler running after a failure.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "347b2776",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 2025-05-06 08:14:15,715 [info] Project loaded successfully: {'project_name': 'waiting-for-multiple-steps'}\n"
     ]
    }
   ],
   "source": [
    "project = mlrun.get_or_create_project(\n",
    "    \"waiting-for-multiple-steps\", \"./\", user_project=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "838bdd4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting funcs.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile funcs.py\n",
    "import time\n",
    "\n",
    "# First 3 functions to run\n",
    "def func_A(context):\n",
    "    time.sleep(5)\n",
    "    context.logger.info(f\"Function A is running now\")\n",
    "    return \"Function A has been triggered\"\n",
    "\n",
    "def func_B(context):\n",
    "    time.sleep(10)\n",
    "    context.logger.info(f\"Function B is running now\")  \n",
    "    # Now it raises an Exception so the function fails.\n",
    "    raise Exception(\"This is an example exception\")\n",
    "\n",
    "def func_C(context):\n",
    "    time.sleep(15)\n",
    "    context.logger.info(f\"Function C is running now\")        \n",
    "    return \"Function C has been triggered\"\n",
    "\n",
    "\n",
    "# This function waits for the 3 functions to complete, then logs their results before raising an exception.\n",
    "def func_D(context, func_a_res, func_b_res, func_c_res):\n",
    "    context.logger.info(f\"Function D is running now, logging the results of previous functions.\")\n",
    "    context.log_artifact(\"func_a_result\", str(func_a_res))\n",
    "    context.log_artifact(\"func_b_result\", str(func_b_res))\n",
    "    context.log_artifact(\"func_c_result\", str(func_c_res))\n",
    "    context.logger.info(\"Function D has been triggered\")\n",
    "    \n",
    "  \n",
    "\n",
    "# This function will be part of the ExitHandler.\n",
    "def func_final(context):\n",
    "    context.logger.info(f\"The final function is now running regardless of whether all preceding functions succeeded.\")\n",
    "    return \"Function final has been triggered\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8ccb572d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the functions in the project\n",
    "func_A = project.set_function(\n",
    "    func=\"funcs.py\", name=\"func-A\", handler=\"func_A\", image=\"mlrun/mlrun\", kind=\"job\"\n",
    ")\n",
    "func_B = project.set_function(\n",
    "    func=\"funcs.py\", name=\"func-B\", handler=\"func_B\", image=\"mlrun/mlrun\", kind=\"job\"\n",
    ")\n",
    "func_C = project.set_function(\n",
    "    func=\"funcs.py\", name=\"func-C\", handler=\"func_C\", image=\"mlrun/mlrun\", kind=\"job\"\n",
    ")\n",
    "func_D = project.set_function(\n",
    "    func=\"funcs.py\", name=\"func-D\", handler=\"func_D\", image=\"mlrun/mlrun\", kind=\"job\"\n",
    ")\n",
    "func_final = project.set_function(\n",
    "    func=\"funcs.py\",\n",
    "    name=\"func-final\",\n",
    "    handler=\"func_final\",\n",
    "    image=\"mlrun/mlrun\",\n",
    "    kind=\"job\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78a8bda8",
   "metadata": {},
   "source": [
    "## Create the pipeline \n",
    "\n",
    "The ExitHandler function runs after the functions, A, B, C, D. \n",
    "It is triggered regardless of whether or not all of the preceding functions succeeded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "858af640",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting workflow.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile workflow.py\n",
    "\n",
    "from kfp import dsl\n",
    "from mlrun.platforms import auto_mount\n",
    "import os\n",
    "import sys\n",
    "import mlrun\n",
    "\n",
    "@dsl.pipeline(name=\"ExitHandler and multiple wait pipeline\", description=\"Pipeline that runs 3 functions simultaneously, waits for all of them to finish, and then runs another function that logs their results and fails. After all of this, an exit function is triggered.\")\n",
    "def kfpipeline(input_val):\n",
    "    project = mlrun.get_current_project()\n",
    "    \n",
    "    # 'func_final' is executed after everything inside the block finishes or crashes.\n",
    "    with dsl.ExitHandler(mlrun.run_function(\"func-final\")):\n",
    "        # Start 3 functions simultaneously\n",
    "        step_1 = mlrun.run_function('func-A', returns=['first_func_res'])\n",
    "        step_2 = mlrun.run_function('func-B', returns=['second_func_res'])\n",
    "        step_3 = mlrun.run_function('func-C', returns=['third_func_res'])\n",
    "\n",
    "        # Start the function only after the 3 first functions are done. This function logs the outputs of the previous functions as artifacts, and then crashes.\n",
    "        step_4 = mlrun.run_function('func-D', params = {\"func_a_res\":step_1.outputs[\"first_func_res\"],\n",
    "                                                        \"func_b_res\":step_2.outputs[\"second_func_res\"],\n",
    "                                                        \"func_c_res\":step_3.outputs[\"third_func_res\"]}, returns=[\"fourth_func_res\"]).after(step_1, step_2, step_3)  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "99fe4c59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mlrun.projects.project.MlrunProject at 0x7f9713208b80>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "project.set_workflow(name=\"workflow-func\", workflow_path=\"workflow.py\")\n",
    "project.save()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01bf2fef-ebc5-4b5b-97cd-06c24d7bf3d5",
   "metadata": {},
   "source": [
    "## Run the workflow\n",
    "\n",
    "Now run the workflow and check the workflow graph in the UI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4592bd72-2396-4ba1-958f-a3a8253de120",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>Pipeline running (id=f8c36890-494f-47f3-9ab7-2b9cd9da7451), <a href=\"https://dashboard.default-tenant.app.cust-cs-il.iguazio-cd0.com/mlprojects/waiting-for-multiple-steps-jill/jobs/monitor-workflows/workflow/f8c36890-494f-47f3-9ab7-2b9cd9da7451\" target=\"_blank\"><b>click here</b></a> to view the details in MLRun UI</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.43.0 (0)\n",
       " -->\n",
       "<!-- Title: kfp Pages: 1 -->\n",
       "<svg width=\"749pt\" height=\"188pt\"\n",
       " viewBox=\"0.00 0.00 748.72 188.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 184)\">\n",
       "<title>kfp</title>\n",
       "<polygon fill=\"white\" stroke=\"transparent\" points=\"-4,4 -4,-184 744.72,-184 744.72,4 -4,4\"/>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1305228400 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1305228400</title>\n",
       "<ellipse fill=\"green\" stroke=\"black\" cx=\"39\" cy=\"-90\" rx=\"38.99\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"39\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">func&#45;a</text>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495</title>\n",
       "<ellipse fill=\"lightgrey\" stroke=\"black\" cx=\"134\" cy=\"-18\" rx=\"39.79\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\">func&#45;d</text>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1305228400&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1305228400&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M59.14,-74.15C72.62,-64.22 90.55,-51.01 105.49,-40.01\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"107.9,-42.57 113.88,-33.82 103.75,-36.94 107.9,-42.57\"/>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1338783638 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1338783638</title>\n",
       "<ellipse fill=\"green\" stroke=\"black\" cx=\"134\" cy=\"-90\" rx=\"38.19\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">func&#45;c</text>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1338783638&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1338783638&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M134,-71.7C134,-63.98 134,-54.71 134,-46.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.5,-46.1 134,-36.1 130.5,-46.1 137.5,-46.1\"/>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1355561257 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1355561257</title>\n",
       "<g id=\"a_node4\"><a xlink:title=\"This is an example exception\">\n",
       "<ellipse fill=\"red\" stroke=\"black\" cx=\"230\" cy=\"-90\" rx=\"39.79\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"230\" y=\"-86.3\" font-family=\"Times,serif\" font-size=\"14.00\">func&#45;b</text>\n",
       "</a>\n",
       "</g>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1355561257&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1355561257&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1389116495</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M209.64,-74.15C195.93,-64.16 177.65,-50.83 162.5,-39.78\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"164.14,-36.65 154,-33.58 160.02,-42.3 164.14,-36.65\"/>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;2644165120 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;2644165120</title>\n",
       "<ellipse fill=\"green\" stroke=\"black\" cx=\"484\" cy=\"-162\" rx=\"256.95\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\">exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6.onExit</text>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782</title>\n",
       "<ellipse fill=\"red\" stroke=\"black\" cx=\"134\" cy=\"-162\" rx=\"75.29\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"134\" y=\"-158.3\" font-family=\"Times,serif\" font-size=\"14.00\">exit&#45;handler&#45;1</text>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1305228400 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1305228400</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M111.96,-144.76C98.58,-134.91 81.38,-122.23 67.03,-111.66\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"69.1,-108.83 58.97,-105.72 64.94,-114.47 69.1,-108.83\"/>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1338783638 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1338783638</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M134,-143.7C134,-135.98 134,-126.71 134,-118.11\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"137.5,-118.1 134,-108.1 130.5,-118.1 137.5,-118.1\"/>\n",
       "</g>\n",
       "<!-- exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1355561257 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;3281838782&#45;&gt;exithandler&#45;and&#45;multiple&#45;wait&#45;pipeline&#45;w9mt6&#45;1355561257</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M156.26,-144.76C169.78,-134.91 187.17,-122.23 201.66,-111.66\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"203.8,-114.44 209.81,-105.72 199.67,-108.78 203.8,-114.44\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f97897af610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h2>Run Results</h2><h3>[info] Workflow f8c36890-494f-47f3-9ab7-2b9cd9da7451 finished with 1 errors</h3><br>click the hyper links below to see detailed results<br><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>uid</th>\n",
       "      <th>start</th>\n",
       "      <th>state</th>\n",
       "      <th>name</th>\n",
       "      <th>parameters</th>\n",
       "      <th>results</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td><div title=\"9803ad2ee40847f6aadf3a216170caff\"><a href=\"https://dashboard.default-tenant.app.cust-cs-il.iguazio-cd0.com/mlprojects/waiting-for-multiple-steps-jill/jobs/monitor/9803ad2ee40847f6aadf3a216170caff/overview\" target=\"_blank\" >...6170caff</a></div></td>\n",
       "      <td>May 06 08:21:13</td>\n",
       "      <td>completed</td>\n",
       "      <td>func-final</td>\n",
       "      <td></td>\n",
       "      <td><div class=\"dictlist\">return=Function final has been triggered</div></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"feff789ef0244c949e4dadcce7af0a08\"><a href=\"https://dashboard.default-tenant.app.cust-cs-il.iguazio-cd0.com/mlprojects/waiting-for-multiple-steps-jill/jobs/monitor/feff789ef0244c949e4dadcce7af0a08/overview\" target=\"_blank\" >...e7af0a08</a></div></td>\n",
       "      <td>May 06 08:20:33</td>\n",
       "      <td><div style=\"color: red;\" title=\"This is an example exception\">error</div></td>\n",
       "      <td>func-b</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"66753d5d47d543189cab9c8d748630e1\"><a href=\"https://dashboard.default-tenant.app.cust-cs-il.iguazio-cd0.com/mlprojects/waiting-for-multiple-steps-jill/jobs/monitor/66753d5d47d543189cab9c8d748630e1/overview\" target=\"_blank\" >...748630e1</a></div></td>\n",
       "      <td>May 06 08:20:33</td>\n",
       "      <td>completed</td>\n",
       "      <td>func-a</td>\n",
       "      <td></td>\n",
       "      <td><div class=\"dictlist\">first_func_res=Function A has been triggered</div></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td><div title=\"430ccb9568664ba58f1f275c5d8b3e65\"><a href=\"https://dashboard.default-tenant.app.cust-cs-il.iguazio-cd0.com/mlprojects/waiting-for-multiple-steps-jill/jobs/monitor/430ccb9568664ba58f1f275c5d8b3e65/overview\" target=\"_blank\" >...5d8b3e65</a></div></td>\n",
       "      <td>May 06 08:20:33</td>\n",
       "      <td>completed</td>\n",
       "      <td>func-c</td>\n",
       "      <td></td>\n",
       "      <td><div class=\"dictlist\">third_func_res=Function C has been triggered</div></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "Pipeline run status Failed",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mproject\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mworkflow-func\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwatch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlocal\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/conda/envs/mlrun-base/lib/python3.9/site-packages/mlrun/projects/project.py:2726\u001b[0m, in \u001b[0;36mMlrunProject.run\u001b[0;34m(self, name, workflow_path, arguments, artifact_path, workflow_handler, namespace, sync, watch, dirty, engine, local, schedule, timeout, source, cleanup_ttl, notifications)\u001b[0m\n\u001b[1;32m   2723\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m engine \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mremote\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m status_engine\u001b[38;5;241m.\u001b[39mengine \u001b[38;5;241m!=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlocal\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m   2724\u001b[0m         status_engine \u001b[38;5;241m=\u001b[39m _RemoteRunner\n\u001b[0;32m-> 2726\u001b[0m     \u001b[43mstatus_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_run_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43mproject\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrun\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2727\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m run\n",
      "File \u001b[0;32m/conda/envs/mlrun-base/lib/python3.9/site-packages/mlrun/projects/pipelines.py:705\u001b[0m, in \u001b[0;36m_KFPRunner.get_run_status\u001b[0;34m(project, run, timeout, expected_statuses, notifiers)\u001b[0m\n\u001b[1;32m    702\u001b[0m notifiers\u001b[38;5;241m.\u001b[39mpush(text, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minfo\u001b[39m\u001b[38;5;124m\"\u001b[39m, runs)\n\u001b[1;32m    704\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m raise_error:\n\u001b[0;32m--> 705\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m raise_error\n\u001b[1;32m    706\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m state, had_errors, text\n",
      "File \u001b[0;32m/conda/envs/mlrun-base/lib/python3.9/site-packages/mlrun/projects/pipelines.py:679\u001b[0m, in \u001b[0;36m_KFPRunner.get_run_status\u001b[0;34m(project, run, timeout, expected_statuses, notifiers)\u001b[0m\n\u001b[1;32m    677\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m timeout:\n\u001b[1;32m    678\u001b[0m         logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWaiting for pipeline run completion\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 679\u001b[0m         state \u001b[38;5;241m=\u001b[39m \u001b[43mrun\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_for_completion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    680\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexpected_statuses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexpected_statuses\u001b[49m\n\u001b[1;32m    681\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    682\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m    683\u001b[0m     \u001b[38;5;66;03m# push runs table also when we have errors\u001b[39;00m\n\u001b[1;32m    684\u001b[0m     raise_error \u001b[38;5;241m=\u001b[39m exc\n",
      "File \u001b[0;32m/conda/envs/mlrun-base/lib/python3.9/site-packages/mlrun/projects/pipelines.py:464\u001b[0m, in \u001b[0;36m_PipelineRunStatus.wait_for_completion\u001b[0;34m(self, timeout, expected_statuses)\u001b[0m\n\u001b[1;32m    463\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait_for_completion\u001b[39m(\u001b[38;5;28mself\u001b[39m, timeout\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, expected_statuses\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m--> 464\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_for_completion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    465\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    466\u001b[0m \u001b[43m        \u001b[49m\u001b[43mproject\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mproject\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    467\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    468\u001b[0m \u001b[43m        \u001b[49m\u001b[43mexpected_statuses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexpected_statuses\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    469\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    470\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state\n",
      "File \u001b[0;32m/conda/envs/mlrun-base/lib/python3.9/site-packages/mlrun/projects/pipelines.py:645\u001b[0m, in \u001b[0;36m_KFPRunner.wait_for_completion\u001b[0;34m(run_id, project, timeout, expected_statuses)\u001b[0m\n\u001b[1;32m    643\u001b[0m     timeout \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m60\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m60\u001b[39m\n\u001b[1;32m    644\u001b[0m project_name \u001b[38;5;241m=\u001b[39m project\u001b[38;5;241m.\u001b[39mmetadata\u001b[38;5;241m.\u001b[39mname \u001b[38;5;28;01mif\u001b[39;00m project \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 645\u001b[0m run_info \u001b[38;5;241m=\u001b[39m \u001b[43mwait_for_pipeline_completion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    646\u001b[0m \u001b[43m    \u001b[49m\u001b[43mrun_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    647\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    648\u001b[0m \u001b[43m    \u001b[49m\u001b[43mexpected_statuses\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mexpected_statuses\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    649\u001b[0m \u001b[43m    \u001b[49m\u001b[43mproject\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mproject_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    650\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    651\u001b[0m status \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    652\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m run_info:\n",
      "File \u001b[0;32m/conda/envs/mlrun-base/lib/python3.9/site-packages/mlrun/run.py:973\u001b[0m, in \u001b[0;36mwait_for_pipeline_completion\u001b[0;34m(run_id, timeout, expected_statuses, namespace, remote, project)\u001b[0m\n\u001b[1;32m    971\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m expected_statuses:\n\u001b[1;32m    972\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m expected_statuses:\n\u001b[0;32m--> 973\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m    974\u001b[0m             \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPipeline run status \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstatus\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;250m \u001b[39mmessage\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mif\u001b[39;00m\u001b[38;5;250m \u001b[39mmessage\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01melse\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    975\u001b[0m         )\n\u001b[1;32m    977\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\n\u001b[1;32m    978\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFinished waiting for pipeline completion.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    979\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m run_id: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrun_id\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    982\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m namespace: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnamespace\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    983\u001b[0m )\n\u001b[1;32m    985\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Pipeline run status Failed"
     ]
    }
   ],
   "source": [
    "project.run(\"workflow-func\", watch=True, local=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "610b2368",
   "metadata": {},
   "source": [
    "## Viewing the results in the UI Monitor Workflows tab\n",
    "\n",
    "<img src=\"../_static/images/multi-step-workflow.png\" width=\"800\" >"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mlrun-base",
   "language": "python",
   "name": "conda-env-mlrun-base-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
